{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Week 2: MLP classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "SVHN can be downloaded from http://ufldl.stanford.edu/housenumbers/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import and preprocess the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from scipy.io import loadmat\n",
    "\n",
    "train = loadmat('../SVHN/train_32x32.mat')\n",
    "test = loadmat('../SVHN/test_32x32.mat')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`train` and `test` are dictionaries with keys `'X'` and `'y'`. The values are numpy arrays."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(train['X'].shape)\n",
    "print(train['y'].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(test['X'].shape)\n",
    "print(test['y'].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "training_set = np.transpose(train['X'], (3, 0, 1, 2)).astype(np.float32)\n",
    "training_labels = train['y']\n",
    "\n",
    "test_set = np.transpose(test['X'], (3, 0, 1, 2)).astype(np.float32)\n",
    "test_labels = test['y']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "n_train = training_set.shape[0]\n",
    "n_test = test_set.shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Inspect the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from matplotlib import pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "example = np.random.choice(np.arange(n_train))\n",
    "\n",
    "image = training_set[example]\n",
    "label = training_labels[example][0]\n",
    "\n",
    "if label == 10:\n",
    "    label = 0\n",
    "\n",
    "plt.imshow(image)\n",
    "plt.show()\n",
    "\n",
    "print(\"Digit: {}\".format(label))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Convert the images to grayscale"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def convert_to_grayscale(images):\n",
    "    images = np.add.reduce(images, keepdims=True, axis=3)\n",
    "    images = images / 3.0\n",
    "    return images / 128.0 - 1.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "training_set_gs = convert_to_grayscale(training_set)\n",
    "test_set_gs = convert_to_grayscale(test_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(training_set_gs.shape)\n",
    "print(test_set_gs.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "example = np.random.choice(np.arange(n_train))\n",
    "\n",
    "image = training_set_gs[example]\n",
    "label = training_labels[example][0]\n",
    "\n",
    "if label == 10:\n",
    "    label = 0\n",
    "\n",
    "plt.imshow(np.squeeze(image), cmap='gray')\n",
    "plt.show()\n",
    "\n",
    "print(\"Digit: {}\".format(label))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Flatten the inputs to feed into an MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_set_flat = training_set_gs.reshape((n_train, -1))\n",
    "test_set_flat = test_set_gs.reshape((n_test, -1))\n",
    "\n",
    "print(training_set_flat.shape)\n",
    "print(test_set_flat.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Encode the labels as one-hot vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def one_hot(labels):\n",
    "    \"\"\"\n",
    "    Encodes the labels as one-hot vectors. Zero is represented as 10 in SVHN.\n",
    "    [10] -> [1, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
    "    [2] -> [0, 0, 1, 0, 0, 0, 0, 0, 0, 0]\n",
    "    \n",
    "    \"\"\"\n",
    "    labels = np.squeeze(labels)\n",
    "    one_hot_labels = []\n",
    "    for num in labels:\n",
    "        one_hot = [0.0] * 10\n",
    "        if num == 10:\n",
    "            one_hot[0] = 1.0\n",
    "        else:\n",
    "            one_hot[num] = 1.0\n",
    "        one_hot_labels.append(one_hot)\n",
    "    labels = np.array(one_hot_labels).astype(np.float32)\n",
    "    return labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "training_labels_one_hot = one_hot(training_labels)\n",
    "test_labels_one_hot = one_hot(test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(training_labels_one_hot.shape)\n",
    "print(test_labels_one_hot.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build the network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class SVHN_MLP:\n",
    "    def __init__(self, wd_factor, learning_rate):\n",
    "        self.wd_factor = wd_factor\n",
    "        self.learning_rate = learning_rate\n",
    "        self.train_pointer = 0\n",
    "        self.test_pointer = 0\n",
    "        \n",
    "        self.sess = tf.Session()\n",
    "        \n",
    "        self.input = tf.placeholder(dtype=tf.float32, shape=[None, 1024], name='input')\n",
    "        self.ground_truth = tf.placeholder(dtype=tf.float32, shape=[None, 10], name='ground_truth')\n",
    "        print(self.input)\n",
    "        \n",
    "        self._build_graph()\n",
    "        \n",
    "    def _build_graph(self):\n",
    "        weights = []  # for weight decay\n",
    "        \n",
    "        with tf.variable_scope('layers'):\n",
    "            h = tf.layers.dense(self.input, 512, kernel_initializer=tf.glorot_uniform_initializer(), \n",
    "                                activation=tf.tanh, name='1')\n",
    "            print(h)\n",
    "            h = tf.layers.dense(h, 256, kernel_initializer=tf.glorot_uniform_initializer(), \n",
    "                                activation=tf.tanh, name='2')\n",
    "            print(h)\n",
    "            h = tf.layers.dense(h, 64, kernel_initializer=tf.glorot_uniform_initializer(), \n",
    "                                activation=tf.tanh, name='3')\n",
    "            print(h)\n",
    "            self.logits = tf.layers.dense(h, 10, kernel_initializer=tf.glorot_uniform_initializer(), \n",
    "                                          activation=tf.identity, name='4')\n",
    "            print(self.logits)\n",
    "            self.prediction = tf.nn.softmax(self.logits, name='softmax_prediction')\n",
    "            \n",
    "        with tf.name_scope('loss'):\n",
    "            self.loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=self.logits, \n",
    "                                                                                  labels=self.ground_truth))\n",
    "            self.loss += self.weight_decay()\n",
    "            \n",
    "        self.optimizer = tf.train.AdamOptimizer(self.learning_rate)\n",
    "        self.train_op = self.optimizer.minimize(self.loss)\n",
    "            \n",
    "    def weight_decay(self):\n",
    "        loss = 0\n",
    "        for v in tf.global_variables():\n",
    "            if 'Adam' in v.name:\n",
    "                continue\n",
    "            elif 'kernel' in v.name:\n",
    "                loss += self.wd_factor * tf.nn.l2_loss(v)\n",
    "        print(loss)\n",
    "        return loss\n",
    "    \n",
    "    def train_minibatch(self, samples, labels, batch_size):\n",
    "        if self.train_pointer + batch_size <= samples.shape[0]:\n",
    "            samples_minibatch = samples[self.train_pointer: self.train_pointer + batch_size]\n",
    "            labels_minibatch = labels[self.train_pointer: self.train_pointer + batch_size]\n",
    "            self.train_pointer += batch_size\n",
    "        else:\n",
    "            samples_minibatch = samples[self.train_pointer:]\n",
    "            labels_minibatch = labels[self.train_pointer: self.train_pointer + batch_size]\n",
    "            self.train_pointer = 0\n",
    "        return samples_minibatch, labels_minibatch\n",
    "\n",
    "    def train(self, train_samples, train_labels, train_batch_size, iteration_steps):\n",
    "        self.sess.run(tf.global_variables_initializer())\n",
    "\n",
    "        print('Start Training')\n",
    "        losses = []\n",
    "        for i in range(iteration_steps):\n",
    "            samples, labels = self.train_minibatch(train_samples, train_labels, train_batch_size)\n",
    "            feed_dict = {self.input: samples, self.ground_truth: labels}\n",
    "            _, loss = self.sess.run([self.train_op, self.loss], feed_dict=feed_dict)\n",
    "            if i % 50 == 0:\n",
    "                print(\"Minibatch loss at step {}: {}\".format(i, loss))\n",
    "                losses.append([i, loss])\n",
    "        return losses\n",
    "                    \n",
    "    def test_minibatch(self, samples, labels, batch_size):\n",
    "        if self.test_pointer + batch_size <= samples.shape[0]:\n",
    "            samples_minibatch = samples[self.test_pointer: self.test_pointer + batch_size]\n",
    "            labels_minibatch = labels[self.test_pointer: self.test_pointer + batch_size]\n",
    "            self.test_pointer += batch_size\n",
    "            end_of_epoch = False\n",
    "        else:\n",
    "            samples_minibatch = samples[self.test_pointer:]\n",
    "            labels_minibatch = labels[self.test_pointer: self.test_pointer + batch_size]\n",
    "            self.test_pointer = 0\n",
    "            end_of_epoch = True\n",
    "        return samples_minibatch, labels_minibatch, end_of_epoch\n",
    "            \n",
    "    def test(self, test_samples, test_labels, test_batch_size):\n",
    "        end_of_epoch = False\n",
    "        losses = []\n",
    "        while not end_of_epoch:\n",
    "            samples, labels, end_of_epoch = self.test_minibatch(test_samples, test_labels, test_batch_size)\n",
    "            feed_dict = {self.input: samples, self.ground_truth: labels}\n",
    "            losses.append(self.sess.run(self.loss, feed_dict=feed_dict))  \n",
    "        print(\"Average test loss: {}\".format(np.mean(losses)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "WD_FACTOR = 0.0001\n",
    "LEARNING_RATE = 0.001\n",
    "model = SVHN_MLP(WD_FACTOR, LEARNING_RATE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.global_variables()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train the network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "TRAIN_BATCH_SIZE = 128\n",
    "ITERATIONS = 10000\n",
    "\n",
    "import time\n",
    "start_time = time.time()\n",
    "\n",
    "losses = model.train(training_set_flat, training_labels_one_hot, TRAIN_BATCH_SIZE, ITERATIONS)\n",
    "\n",
    "end_time = time.time()\n",
    "print(\"Training time: {}s\".format(end_time - start_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "losses = np.array(losses)\n",
    "print(losses.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "iterations = losses[:, 0]\n",
    "train_loss = losses[:, 1]\n",
    "plt.figure(figsize=(10, 5))\n",
    "plt.plot(iterations, train_loss)\n",
    "plt.xlabel(\"Iterations\")\n",
    "plt.ylabel(\"Loss\")\n",
    "plt.title(\"Training curve\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test network predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "TEST_BATCH_SIZE = 128\n",
    "\n",
    "model.test(test_set_flat, test_labels_one_hot, TEST_BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "example = np.random.choice(np.arange(n_test))\n",
    "\n",
    "sample = np.expand_dims(test_set_flat[example], axis=0)\n",
    "label = np.expand_dims(test_labels_one_hot[example], axis=0)\n",
    "\n",
    "digit = np.where(label[0]==1.0)[0][0]\n",
    "\n",
    "feed_dict = {model.input: sample, model.ground_truth: label}\n",
    "prediction = model.sess.run(model.prediction, feed_dict=feed_dict)[0]\n",
    "\n",
    "image = np.reshape(sample, (32, 32))\n",
    "\n",
    "print(\"Test sample digit: {}\".format(digit))\n",
    "fig, ax = plt.subplots(1, 2, figsize=(17, 5))\n",
    "ax[0].imshow(image, cmap='gray')\n",
    "ax[0].set_title(\"Test example\")\n",
    "\n",
    "classes = np.arange(10)\n",
    "width = 1.0\n",
    "\n",
    "#fig, ax = plt.subplots()\n",
    "ax[1].bar(classes, prediction, width, color='Blue')\n",
    "ax[1].set_ylabel('Probabilities')\n",
    "ax[1].set_title('Network categorical distribution')\n",
    "ax[1].set_xticks(classes)\n",
    "ax[1].set_xticklabels(('0', '1', '2', '3', '4', '5', '6', '7', '8', '9'))\n",
    "ax[1].set_xlabel('Digit class')\n",
    "\n",
    "plt.show()\n",
    "\n",
    "print(\"Network prediction probabilities:\")\n",
    "print(prediction)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.sess.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
